IRIX Installation Tools & Overlays 2001 November/SGI IRIX Installation Tools & Overlays 2001 November - Disc 3.iso/dist/cluster_services.idb/var/cluster/cmgr-scripts/haStatus.z/haStatus

Text File | 2001-10-10 | 32KB | 1,247 lines

#!/bin/ksh
#
#  Copyright (C) 1998, Silicon Graphics, Inc.
#  All Rights Reserved.
#
#  UNPUBLISHED -- Rights reserved under the copyright laws of the United
#  States.  Use of a copyright notice is precautionary only and does not
#  imply publication or disclosure.
#
#  THIS SOFTWARE CONTAINS CONFIDENTIAL AND PROPRIETARY INFORMATION OF
#  SILICON GRAPHICS, INC. ANY DUPLICATION, MODIFICATION, DISTRIBUTION, OR
#  DISCLOSURE IS STRICTLY PROHIBITED WITHOUT THE PRIOR EXPRESS WRITTEN
#  PERMISSION OF SILICON GRAPHICS, INC.
#
#  U.S. GOVERNMENT RESTRICTED RIGHTS LEGEND
#  Use, duplication or disclosure by the Government is subject to
#  restrictions as set forth in FAR 52.227.19(c)(2) or subparagraph
#  (c)(1)(ii) of the Rights in Technical Data and Computer Software
#  clause at DFARS 252.227-7013 and/or in similar or successor clauses
#  in the FAR, or the DOD or NASA FAR Supplement.  Unpublished-- rights
#  reserved under the copyright laws of the United States.
#  Contractor/manufacturer is Silicon Graphics, Inc.,
#  2011 N. Shoreline Blvd., Mountain View, CA 94039-7311.
#
#
# haStatus - This scripts prints the current state of all clusters, their
#            nodes and their resource groups as configured in the cdb of
#            the node on which this script is run.
#         Detailed configuration information for all clusters, their
#            nodes, resource groups, resources and failover policies, and for
#            nodes which are in the pool but not in any cluster can be
#            printed using the -i option.
#            Using option -a all the information printed with -i option can
#            be printed along with status information.
#            See comments on USAGE below for details.

SCRIPT_NAME=`/usr/bin/basename $0`


#
# The usage string for this script.
#
# A brief status information is printed when no option is given. This includes:
# 1. The cluster names and their status.
# 2. The nodes in each cluster and their status.
# 3. The resource groups in each cluster, their status and their resources.
#
# When -i option is used for this script, detailed configuration information
# pertaining to all clusters in the cdb of the node on which this script is
# run is printed:
# 1. For each cluster defined in cdb, following information is printed:
#    a. cluster name
#    b. cluster notify cmd
#    c. cluster notify address(es).
#    d. The nodes in the cluster, their status, and their configuration
#       information as printed by cmgr command show node A
#    e. The resource groups in the cluster, their status, their failover
#       policy, and their resources.
#    f. A list of resources in the cluster with their state and their
#       configuration information as printed by cmgr command show resource
#       A of resource_type B.
# 2. A list of failover policies in the cdb and their configuration
#    information as printed by cmgr command show failover_policy A in cluster B.
# 3. A list of nodes in the pool but not included in any cluster and their
#    configuration information.
#
# The user can use option -a to all configuration information as printed by
# option -i and status information for each item listed above for option -i
# if applicable.
#
# The user can use option -c to print information for a specific cluster.
# Information pertaining to only 1 cluster will be printed.
#
USAGESTR="\
Usage: $SCRIPT_NAME [-a|-i] [-c clustername] \n\
where,\n\
 -a prints detailed cluster configuration information and cluster status.\n\
 -i prints detailed cluster configuration information only.\n\
 -c can be used to specify a cluster for which status is to be printed.\n\
 \"clustername\" is the name of the cluster for which status is to be printed.\
"


#
# CMGR_COMMAND_INTERPRETER - Full path name of the cluster_mgr application.
# ** NOTE TO THE USER ** If cluster_mgr application is installed in non-default
# location modify this variable to indicate the correct path
#
CMGR_COMMAND_INTERPRETER="/usr/cluster/bin/cluster_mgr"



# Temporary files needed by the script

#
# CMGR_OUTPUT - The output from the cmgr commands is saved in this file.
#
CMGR_OUTPUT=/tmp/${SCRIPT_NAME}$$
CMGR_ERROR=/tmp/${SCRIPT_NAME}$$.err
CMGR_OUTPUT1=/tmp/${SCRIPT_NAME}$$.1
CMGR_ERROR1=/tmp/${SCRIPT_NAME}$$.err.1

#
# TMPFILE - Needed by function prependtab().
#
TMPFILE=/tmp/${SCRIPT_NAME}$$.tmp

#
# SHOW_STATUS - Flag to indicate whether status information should be printed.
#               Values: 0 - show status,   1 - do not show status
#               Default value: 0  (show status).
#
SHOW_STATUS=0

#
# SHOW_DETAILED_CONFIG - Flag to indicate whether detailed configuration
#                        information should be printed.
#               Values: 0 - show detailed,   1 - show brief
#               Default value: 1  (show brief).
#
SHOW_DETAILED_CONFIG=1


#
# SHOW_ALL - Set to 0 when user specifies -a option to print detailed cluster
#            configuration information and status.
#
SHOW_ALL=1

#
# SHOW_CONFIG_ONLY - Set to 0 when user specifies -i option to print only the
#                    detailed cluster configuration information. No status
#                    information is printed. -a option overrides -i option.
#
SHOW_CONFIG_ONLY=1


#############
# FUNCTIONS #
#############

#
# signal_handler():
#     Arguments: None
#    Return value: None
#    Description:
#         Delete the temporary files and terminates this script. Invoked
#               when user interrupts the script using signal 2 (SIGTERM).
#
function signal_handler
{

    rm -f $CMGR_OUTPUT
    rm -f $CMGR_ERROR
    rm -f $CMGR_OUTPUT1
    rm -f $CMGR_ERROR1
    rm -f $TMPFILE

    exit 1
}


#
# run_cmgr_command():
#     Argument1: cmd - cluster_mgr command to be executed with cluster_mgr's
#                        -c option
#    Return Value: 0 always
#    Description:
#         Executes the cluster_mgr command.
#        Deletes any blank lines from the output.
#        Saves the result in $CMGR_OUTPUT file.
#        Saves the stderr in $CMGR_ERROR file.
#
function run_cmgr_command
{
    cmd=$*

    rm -f $CMGR_OUTPUT
    if [ -a $CMGR_OUTPUT ]; then
        echo -n "Error: $SCRIPT_NAME: Couldn't remove "
        echo    "temporary file ${CMGR_OUTPUT}."
        exit 1
    fi

    rm -f $CMGR_ERROR
    if [ -a $CMGR_ERROR ]; then
        echo -n "Error: $SCRIPT_NAME: Couldn't remove "
        echo    "temporary file ${CMGR_ERROR}."
        exit 1
    fi


    # Remove any blank lines from the cmgr output before saving it in
    # $CMGR_OUTPUT file.
    ${CMGR_COMMAND_INTERPRETER} -c "$cmd" 2>$CMGR_ERROR | \
            sed -e '/^[   ]*$/d' > $CMGR_OUTPUT 2>/dev/null

    return 0

}


#
# prependtab():
#     Argument1: filename - Name of the a text file
#    Return Value: 0 always
#    Description:
#         Adds eight spaces at the beginning of each line in file
#        filename.
#
function prependtab
{
    filename=$1

    rm -f $TMPFILE

        if [ -a $TMPFILE ]; then
                echo -n "Error: $SCRIPT_NAME: Couldn't remove temporary "
        echo    "file ${TMPFILE}."
                exit 1
        fi

    sed -n -e 's/^/        /p' $filename > $TMPFILE
    mv -f $TMPFILE $filename
    return 0
}



#
# get_clusters():
#     Arguments: None
#    Input: None
#    Return Value: 0 on success, 1 on failure.
#    Output: Array CLUSTER[] and Variable NUMCLUSTERS
#    Description:
#        Obtains list of clusters defined in cdb using cmgr command
#               show clusters. Saves the cluster names in array CLUSTER[]
#               and the number of clusters in variable NUMCLUSTERS.
#
function get_clusters
{
    # Obtain the list of clusters configured using cmgr show clusters.

    run_cmgr_command show clusters

    # Syntax of the output of the show clusters command when successful is:
    #    N Cluster(s) defined
    #        clustername1
    #        clustername2
    #        ....
    #        clusternameN

    # When <= 1 line in the $CMGR_OUTPUT file, it implies that cmgr show
    # clusters was unsuccessful.
    NUMLINES=`cat $CMGR_OUTPUT | wc -l`
    if [ $NUMLINES -le 1 ]; then
        echo "Error: $SCRIPT_NAME: Could not get the list of clusters."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi


    # Confirm that from the second line to the end of file, you have a
    # list of clusters, by confirming that there is 1 word in each
    # non-blank line. Otherwise, it can be safely assumed to be an error
    # message.
    while [ $NUMLINES -gt 1 ]
    do
        NUMWORDS=`cat $CMGR_OUTPUT | sed -n -e "${NUMLINES}p" | wc -w`
        if [ $NUMWORDS -gt 1 ]; then
            echo -n "Error: $SCRIPT_NAME: Could not get "
            echo    "the list of clusters."
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            return 1
        fi

        NUMLINES=`expr $NUMLINES - 1`
    done

    # Save the clusters in the CLUSTER array and number of clusters in
    # NUMCLUSTERS variable.
    clusteridx=0
    for cluster in `cat $CMGR_OUTPUT | sed -e '1d'`
    do
        CLUSTER[$clusteridx]=$cluster
        clusteridx=`expr $clusteridx + 1`
    done

    NUMCLUSTERS=$clusteridx

    # Make sure that clusters were obtained.
    if [ $NUMCLUSTERS -le 0 ]; then
        echo "Error: $SCRIPT_NAME: Could not get the list of clusters."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    cat $CMGR_ERROR

    rm -f $CMGR_OUTPUT
    rm -f $CMGR_ERROR

    return 0
}


#
# get_cluster_nodes():
#     Arguments: None
#    Input: Variable CLUSTER_NAME containing the name of the cluster.
#    Return Value: 0 on success, 1 on failure
#    Output: Array NODE[] and Variable NUMNODES
#    Description:
#        Obtains list of nodes in cluster $CLUSTER_NAME.
#               Saves the node names in array NODE[] and the number of
#               nodes in variable NUMNODES.
#
function get_cluster_nodes
{
    run_cmgr_command show nodes in cluster $CLUSTER_NAME

    # Syntax of the $CMGR_OUTPUT file when above command successful is
    #    Cluster test-cluster has following N machine(s)
    #        nodename1
    #        nodename2
    #        ....
    #        nodenameN

    # When <= 1 line in the $CMGR_OUTPUT file, it implies that cmgr show
    # nodes in cluster was unsuccessful.
    NUMLINES=`cat $CMGR_OUTPUT | wc -l`
    if [ $NUMLINES -le 1 ]; then
        echo -n "Error: $SCRIPT_NAME: Could not determine the nodes "
        echo    "in the cluster ${CLUSTER_NAME}."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi


    # Confirm that from the second line to the end of file, you have a
    # list of nodes, by confirming that there is 1 word in each non-blank
    # line. Otherwise, it can be safely assumed to be an error message.
    while [ $NUMLINES -gt 1 ]
    do
        NUMWORDS=`cat $CMGR_OUTPUT | sed -n -e "${NUMLINES}p" | wc -w`
        if [ $NUMWORDS -gt 1 ]; then
            echo -n "Error: $SCRIPT_NAME: Could not determine "
            echo    "the nodes in the cluster ${CLUSTER_NAME}."
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            return 1
        fi

        NUMLINES=`expr $NUMLINES - 1`
    done

    # Save the nodes in the NODE array and number of nodes in NUMNODES
    # variable.
    nodeidx=0
    for node in `cat $CMGR_OUTPUT | sed -e '1d'`
    do
        NODE[$nodeidx]=$node
        nodeidx=`expr $nodeidx + 1`
    done
    NUMNODES=$nodeidx

    # Make sure that NODES were obtained.
    if [ $NUMNODES -le 0 ]; then
        echo -n "Error: $SCRIPT_NAME: Could not determine the nodes "
        echo    "in the cluster ${CLUSTER_NAME}."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    # Append the list of nodes to CNODES[] array. This information will
    # be later used to determine the nodes which are in the pool but are
    # not in any cluster.
    nodeidx=0
    cnodeidx=$NUMCNODES
    while [ $nodeidx -lt $NUMNODES ]
    do
        CNODE[$cnodeidx]=${NODE[$nodeidx]}
        cnodeidx=`expr $cnodeidx + 1`
        nodeidx=`expr $nodeidx + 1`
    done
    NUMCNODES=$cnodeidx

    cat $CMGR_ERROR

    rm -f $CMGR_OUTPUT
    rm -f $CMGR_ERROR

    return 0
}


#
# get_resource_groups_in_cluster():
#     Arguments: None
#    Input: Variable CLUSTER_NAME containing the name of the cluster.
#    Return Value: 0 on success, 1 on failure
#    Output: Array GROUP[] and Variable NUMGROUPS
#    Description:
#        Obtains list of resource groups in cluster $CLUSTER_NAME.
#               Saves the resource group names in array GROUP[] and the
#               number of resource groups in variable NUMGROUPS.
#
function get_resource_groups_in_cluster
{
    run_cmgr_command show resource_groups in cluster $CLUSTER_NAME

    # Syntax of the $CMGR_OUTPUT file when the show resource_groups command
    # above is successful
    #
    #    Resource Groups:
    #        resource_group1
    #        resource_group2
    #        ....
    #        resource_groupN

    # When <= 1 line in the $CMGR_OUTPUT file, it implies that
    # show resource_groups in cluster was unsuccessful.
    NUMLINES=`cat $CMGR_OUTPUT | wc -l`
    if [ $NUMLINES -le 1 ]; then
        echo -n "Warning: $SCRIPT_NAME: Could not determine "
        echo    "resource groups in the cluster ${CLUSTER_NAME}."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        echo " "; echo " "
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi


    # Confirm that from the second line to the end of file, you have
    # a list of resource groups, by confirming that there is 1 word in
    # each non-blank line. Otherwise, it can be safely assumed to be
    # an error message.
    while [ $NUMLINES -gt 1 ]
    do
        NUMWORDS=`cat $CMGR_OUTPUT | sed -n -e "${NUMLINES}p" | wc -w`
        if [ $NUMWORDS -gt 1 ]; then
            echo -n "Warning: $SCRIPT_NAME: Could not determine "
            echo -n "resource groups in the cluster "
            echo    "${CLUSTER_NAME}."
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            echo " "; echo " "
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            return 1
        fi

        NUMLINES=`expr $NUMLINES - 1`
    done

    # Save the resource_group names in the GROUP[] array, and number
    # of resource groups in NUMGROUPS variable
    grpidx=0
    for resgroup in `cat $CMGR_OUTPUT | sed -e '1d'`
    do
        GROUP[$grpidx]=$resgroup
        grpidx=`expr $grpidx + 1`
    done

    NUMGROUPS=$grpidx

    # Confirm that the resource groups were obtained.
    if [ $NUMGROUPS -eq 0 ]; then
        echo -n "Warning: $SCRIPT_NAME: Could not determine "
        echo    "resource groups in the cluster ${CLUSTER_NAME}."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        echo " "; echo " "
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    cat $CMGR_ERROR

    rm -f $CMGR_OUTPUT
    rm -f $CMGR_ERROR

    return 0
}


#
# get_and_print_resources_in_cluster():
#     Arguments: None
#    Input: Variable CLUSTER_NAME containing the name of the cluster.
#    Return Value: 0 on success, 1 on failure
#    Output:None
#    Description:
#        Obtains list of resources in cluster $CLUSTER_NAME by
#        first obtaining a list of resource_types in the cluster
#        and then all resources of each resource_type in the cluster.
#               Prints the status (if requested) and details of each resource.
#
function get_and_print_resources_in_cluster
{
    run_cmgr_command show resource_types in cluster $CLUSTER_NAME

    # Syntax of the $CMGR_OUTPUT file when the show resource_types command
    # above is successful
    #
    #        resource_type1
    #        resource_type2
    #        ....
    #        resource_typeN


    # Confirm that from the first line to the end of file, you have
    # a list of resource types, by confirming that there is 1 word in
    # each non-blank line. Otherwise, it can be safely assumed to be
    # an error message.
    NUMLINES=`cat $CMGR_OUTPUT | wc -l`
    while [ $NUMLINES -gt 0 ]
    do
        NUMWORDS=`cat $CMGR_OUTPUT | sed -n -e "${NUMLINES}p" | wc -w`
        if [ $NUMWORDS -gt 1 ]; then
            echo -n "Warning: $SCRIPT_NAME: Could not determine "
            echo -n "resource types in the cluster "
            echo    "${CLUSTER_NAME}."
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            echo " "; echo " "
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            return 1
        fi

        NUMLINES=`expr $NUMLINES - 1`
    done

    cat $CMGR_ERROR

    rm -f $CMGR_ERROR

    # Obtain the resources of each resource_type in the cluster
    NUMRESOURCES=0
    for restype in `cat $CMGR_OUTPUT`
    do
        run_cmgr_command show resources of resource_type $restype \
                        in cluster $CLUSTER_NAME


        # Syntax of the $CMGR_OUTPUT file when the show resources of
        # resource_type command above is successful
        #
        #        res1 type resource_type
        #        res2 type resource_type
        #        ....
        #        resN type resource_type

        # Save the CMGR_OUTPUT and CMGR_ERROR files
        rm -f $CMGR_OUTPUT1
        if [ -a $CMGR_OUTPUT1 ]; then
            echo -n "Error: $SCRIPT_NAME: Couldn't remove "
            echo    "temporary file ${CMGR_OUTPUT1}."
            exit 1
        fi
        mv -f $CMGR_OUTPUT $CMGR_OUTPUT1

        rm -f $CMGR_ERROR1
        if [ -a $CMGR_ERROR1 ]; then
            echo -n "Error: $SCRIPT_NAME: Couldn't remove "
            echo    "temporary file ${CMGR_ERROR1}."
            exit 1
        fi
        mv -f $CMGR_ERROR $CMGR_ERROR1

        # Confirm that from the first line to the end of file, you have
        # a list of resources, by confirming that there are 3 words in
        # each line, where the second and 3rd words are
        # "type" and $restype respectively.
        # Otherwise, it can be safely assumed to be an error message.
        NUMLINES=`cat $CMGR_OUTPUT1 | wc -l`
        while [ $NUMLINES -gt 0 ]
        do
            LINE=`cat $CMGR_OUTPUT1 | sed -n -e "${NUMLINES}p"`
            NUMWORDS=`echo $LINE | wc -w`
            if [ $NUMWORDS -ne 3 ]; then
                echo -n "Warning: $SCRIPT_NAME: Could not "
                echo -n "determine resources of resource_type "
                echo -n "$restype in the cluster "
                echo    "${CLUSTER_NAME}."
                cat $CMGR_ERROR1
                cat $CMGR_OUTPUT1
                echo " "; echo " "
                rm -f $CMGR_OUTPUT1
                rm -f $CMGR_ERROR1
                return 1
            fi

            SECOND_AND_THIRD_WORD=`echo $LINE | awk '{print $2 $3}'`
            if [ "a$SECOND_AND_THIRD_WORD" != \
                        "atype$restype" ]; then
                echo -n "Warning: $SCRIPT_NAME: Could not "
                echo -n "determine resources of resource_type "
                echo -n "$restype in the cluster "
                echo    "${CLUSTER_NAME}."
                cat $CMGR_ERROR1
                cat $CMGR_OUTPUT1
                echo " "; echo " "
                rm -f $CMGR_OUTPUT1
                rm -f $CMGR_ERROR1
                return 1
            fi


            # Get and print status of the resource and its details.
            RESOURCE="`echo $LINE | awk '{print $1}'`"
            echo "Resource $RESOURCE (type ${restype}):"
            echo " "

            # Print resource status.
            if [ $SHOW_STATUS -eq 0 ]; then
                run_cmgr_command show status of resource \
                    $RESOURCE of resource_type $restype \
                        in cluster $CLUSTER_NAME
                prependtab $CMGR_OUTPUT
                cat $CMGR_ERROR
                cat $CMGR_OUTPUT
                rm -f $CMGR_OUTPUT
                rm -f $CMGR_ERROR
                echo " "
            fi


            # Print resource details.
            run_cmgr_command show resource $RESOURCE \
                        of resource_type $restype
            prependtab $CMGR_OUTPUT
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            echo " "

            NUMRESOURCES=`expr $NUMRESOURCES + 1`

            echo " "

            NUMLINES=`expr $NUMLINES - 1`

        done
        cat $CMGR_ERROR1
        rm -f $CMGR_OUTPUT1
        rm -f $CMGR_ERROR1
    done


    # Confirm that the resources were obtained.
    if [ $NUMRESOURCES -le 0 ]; then
        echo -n "Warning: $SCRIPT_NAME: Could not determine resources "
        echo    "in the cluster ${CLUSTER_NAME}."
        echo " "; echo " "
        return 1
    fi

    return 0
}



#
# get_failover_policies():
#     Arguments: None
#    Input: None
#    Return Value: 0 on success, 1 on failure
#    Output: Array  FAILOVER_POLICY[] and Variable NUMPOLICIES
#    Description:
#        Obtains list of failover policies defined in cdb using
#               cmgr command show failover_policies.
#               Saves the failover policy names in array FAILOVER_POLICY[]
#               and the number of failover policies in variable NUMPOLICIES.
#
function get_failover_policies
{

    run_cmgr_command show failover_policies

    # Syntax of the output of the show failover_policies which is saved in
    # $CMGR_OUTPUT file:
    #
    #    Failover Policies:
    #        failover-policy1
    #        failover-policy2
    #        ....
    #        failover-policyN

    # When <= 1 line in the $CMGR_OUTPUT file, it implies that show
    # failover_policies command was unsuccessful.
    NUMLINES=`cat $CMGR_OUTPUT | wc -l`
    if [ $NUMLINES -le 1 ]; then
        echo -n "Warning: $SCRIPT_NAME: Could not determine "
        echo    "the failover_policies."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        echo " "; echo " "
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    # Confirm that from the second line to the end of file, you have a list
    # of failover policies, by confirming that there is 1 word in each
    # non-blank line. Otherwise, it can be safely assumed to be an error
    # message.
    while [ $NUMLINES -gt 1 ]
    do
        NUMWORDS=`cat $CMGR_OUTPUT | sed -n -e "${NUMLINES}p" | wc -w`
        if [ $NUMWORDS -gt 1 ]; then
            echo -n "Warning: $SCRIPT_NAME: Could not determine "
            echo    "the failover_policies."
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            echo " "; echo " "
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            return 1
        fi

        NUMLINES=`expr $NUMLINES - 1`
    done

    # Save the failover policies in array FAILOVER_POLICY[] and number of
    # failover policies in variable NUMPOLICIES
    policyidx=0
    for policy in `cat $CMGR_OUTPUT | sed -e '1d'`
    do
        FAILOVER_POLICY[$policyidx]=$policy
        policyidx=`expr $policyidx + 1`
    done

    NUMPOLICIES=$policyidx

    # Confirm that failover policies were obtained.
    if [ $NUMPOLICIES -eq 0 ]; then
        echo -n "Warning: $SCRIPT_NAME: Could not determine "
        echo    "the failover_policies."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        echo " "; echo " "
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    cat $CMGR_ERROR

    rm -f $CMGR_OUTPUT
    rm -f $CMGR_ERROR
    return 0
}


#
# get_pool_nodes():
#       Arguments: None
#       Input: None
#       Return Value: 0 on success, 1 on failure
#       Output: Array NODE[] and Variable NUMNODES
#       Description:
#               Obtains list of nodes in the pool.
#               Saves the node names in array NODE[] and the number of
#               nodes in variable NUMNODES.
#
function get_pool_nodes
{

    run_cmgr_command show nodes in pool

    # Syntax of the output of the show nodes in pool which is saved in
    # $CMGR_OUTPUT file:
    #
    #    N Machine(s) defined
    #            node1
    #        node2
    #        ...
    #        nodeN


    # When <= 1 line in the $CMGR_OUTPUT file, it implies that show
    # nodes in pool command was unsuccessful.
    NUMLINES=`cat $CMGR_OUTPUT | wc -l`
    if [ $NUMLINES -le 1 ]; then
        echo -n "Error: $SCRIPT_NAME: Could not determine "
        echo    "the nodes in the pool."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    # Confirm that from the second line to the end of file, you have a list
    # of nodes, by confirming that there is 1 word in each non-blank line.
    # Otherwise, it can be safely assumed to be an error message.
    while [ $NUMLINES -gt 1 ]
    do
        NUMWORDS=`cat $CMGR_OUTPUT | sed -n -e "${NUMLINES}p" | wc -w`
        if [ $NUMWORDS -gt 1 ]; then
            echo -n "Error: $SCRIPT_NAME: Could not determine "
            echo    "the nodes in the pool."
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            return 1
        fi

        NUMLINES=`expr $NUMLINES - 1`
    done

    # Save the nodes in array NODE[] and number of nodes in
    # NUMNODES variable.
    nodeidx=0
    for node in `cat $CMGR_OUTPUT | sed -e '1d'`
    do
        NODE[$nodeidx]=$node
        nodeidx=`expr $nodeidx + 1`
    done

    NUMNODES=$nodeidx

    # Confirm that nodes were obtained.
    if [ $NUMNODES -le 0 ]; then
        echo -n "Error: $SCRIPT_NAME: Could not determine "
        echo    "the nodes in the pool."
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        return 1
    fi

    cat $CMGR_ERROR

    rm -f $CMGR_OUTPUT
    rm -f $CMGR_ERROR
    return 0
}


################
# MAIN SECTION #
################


#
# When interrupted by user, remove temporary files and exit
#
trap signal_handler 2


#
# Process the command line arguments
#
while getopts aic:h  c
do
    case $c in
        a)        SHOW_ALL=0;;
        i)        SHOW_CONFIG_ONLY=0;;
        c)        CLUSTER_NAME=$OPTARG;;
        h | \?)        echo $USAGESTR
                exit 1;;
    esac

done

echo " "
date
echo " "

# Option -a always overrides option -i.
if [ $SHOW_ALL -eq 0 ]; then
    SHOW_STATUS=0
    SHOW_DETAILED_CONFIG=0
elif [ $SHOW_CONFIG_ONLY -eq 0 ]; then
    SHOW_STATUS=1
    SHOW_DETAILED_CONFIG=0
fi

# If the user specified a cluster name, save it as the the only cluster
# in the list of clusters and proceed. Else, obtain the list of clusters.
if [ "a$CLUSTER_NAME" != "a" ]; then
    CLUSTER[0]=$CLUSTER_NAME
    NUMCLUSTERS=1
else
    get_clusters
    if [ $? -ne 0 ]; then
        # No cluster names could be obtained. Exit.
        # Error message is printed and cleanup is done in get_clusters
        exit 1
    fi
fi

echo " "
clusteridx=0
# At the end of following while loop NUMCNODES will contain the
# number of nodes in all cluster if detailed configuration
# information is requested. CNODE[] will have a list of nodes in
# all clusters.
NUMCNODES=0 # Updated by get_cluster_nodes
while [ $clusteridx -lt $NUMCLUSTERS ]
do

    CLUSTER_NAME=${CLUSTER[$clusteridx]}


    echo "Cluster ${CLUSTER_NAME}:"
    echo " "



    # Print status of cluster
    if [ $SHOW_STATUS -eq 0 ]; then
        run_cmgr_command show status of cluster $CLUSTER_NAME
        prependtab $CMGR_OUTPUT
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        echo " "
    fi


    # If detailed configuration information requested, print
    # Cluster Notify Cmd and Address

    if [ $SHOW_DETAILED_CONFIG -eq 0 ]; then
        run_cmgr_command show cluster $CLUSTER_NAME
        prependtab $CMGR_OUTPUT
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT | sed -n -e '/Cluster Notify/p'
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        echo " "
    fi


    echo " "


    #
    # Obtain the nodes in the cluster in NODE[] array and number of
    # nodes in NUMNODES variable.
    #
    get_cluster_nodes
    if [ $? -ne 0 ]; then
        # Nodes could not be found skip to next cluster.
        # Error message printed and any cleanup done by
        # function get_cluster_nodes.
        echo " "
        echo " ...skipping cluster $CLUSTER_NAME"
        echo " "
        clusteridx=`expr $clusteridx + 1`
        continue
    fi


    #
    # Get and print the status of each node.
    #

    #if [ $NUMNODES -eq 1 ]; then
    #    echo The $NUMNODES node in the cluster and its state is:
    #else
    #    echo The $NUMNODES nodes in the cluster and their states are:
    #fi
    #echo " "

    nodeidx=0
    while [ $nodeidx -lt $NUMNODES ]
    do
        echo "Node ${NODE[$nodeidx]}:"
        echo " "

        # Print status of node.
        if [ $SHOW_STATUS -eq 0 ]; then
            run_cmgr_command show status of node ${NODE[$nodeidx]}
            prependtab $CMGR_OUTPUT
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            echo " "
        fi

        # If user has requested detailed information, show details
        # of each node
        if [ $SHOW_DETAILED_CONFIG -eq 0 ]; then
            run_cmgr_command show node ${NODE[$nodeidx]}
            prependtab $CMGR_OUTPUT
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            echo " "
        fi


        echo " "

        nodeidx=`expr $nodeidx + 1`
    done

    #
    # Obtain the resource groups in the cluster.
    #
    get_resource_groups_in_cluster
    if [ $? -eq 0 ]; then  #resource_groups found
        #
        # Get and print the status of each resource group.
        #

        grpidx=0
        while [ $grpidx -lt $NUMGROUPS ]
        do
            echo "Resource_group ${GROUP[$grpidx]}:"
            echo " "

            # Show resource group status.
            if [ $SHOW_STATUS -eq 0 ]; then
                run_cmgr_command show status of resource_group \
                    ${GROUP[$grpidx]} in cluster \
                        $CLUSTER_NAME
                prependtab $CMGR_OUTPUT
                cat $CMGR_ERROR
                cat $CMGR_OUTPUT
                rm -f $CMGR_OUTPUT
                rm -f $CMGR_ERROR
                echo " "
            fi

            # Always show the resource group's failover policy
            run_cmgr_command show resource_group \
                ${GROUP[$grpidx]} in cluster $CLUSTER_NAME
            # Get failover_policy name
            fp_name=`awk -F: \
                '$1 ~ /Failover Policy/ {printf ("%s", $2)}' \
                    $CMGR_OUTPUT | sed -e 's/\ //g'`
            if [ "a$fp_name" = "a" ]; then
                        echo -n "Warning: $SCRIPT_NAME: Could not "
                echo -n "determine the failover_policy "
                echo    "for this resource_group."
                cat $CMGR_ERROR
                cat $CMGR_OUTPUT
                rm -f $CMGR_OUTPUT
                rm -f $CMGR_ERROR
            else

                cat $CMGR_ERROR

                rm -f $CMGR_OUTPUT
                rm -f $CMGR_ERROR

                # If the user requested detailed information
                # obtain the failover policy info
                if [ $SHOW_DETAILED_CONFIG -eq 0 ]; then
                    run_cmgr_command show failover_policy \
                                ${fp_name}
                    prependtab $CMGR_OUTPUT
                fi

                # Print the failover_policy name
                echo "\tFailover Policy: $fp_name"

                # Now print the failover policy info
                # if the user requested detailed information
                if [ $SHOW_DETAILED_CONFIG -eq 0 ]; then
                    cat $CMGR_ERROR
                    cat $CMGR_OUTPUT | sed -e \
                        "/Failover Policy: ${fp_name}/d"
                    rm -f $CMGR_OUTPUT
                    rm -f $CMGR_ERROR
                fi
            fi

            echo " "

            # Always show the resources in the resource group.
            run_cmgr_command show resources in resource_group \
                ${GROUP[$grpidx]} in cluster $CLUSTER_NAME
            prependtab $CMGR_OUTPUT
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR

            echo " "
            echo " "


            grpidx=`expr $grpidx + 1`
        done
    fi

    # If the user has requested detailed cluster information, obtain and
    # print the resources in the cluster and their status(optional)
    # and details.
    if [ $SHOW_DETAILED_CONFIG -eq 0 ]; then
        get_and_print_resources_in_cluster
    fi


    # Proceed to the next cluster
    clusteridx=`expr $clusteridx + 1`
    if [ $clusteridx -lt $NUMCLUSTERS ]; then
        echo " "
    fi

done

#
# If detailed configuration information is requested:
#

if [ $SHOW_DETAILED_CONFIG -eq 0 ]; then

# Show the failover_policies

    # Obtain the failover-policies
    get_failover_policies
    if [ $? -eq 0 ]; then     # Failover policies could be found

        #
        # Print the failover policies and their details.
        #
        policyidx=0
        while [ $policyidx -lt $NUMPOLICIES ]
        do
            echo "Failover_policy ${FAILOVER_POLICY[$policyidx]}:"
            echo " "

            run_cmgr_command show failover_policy \
                ${FAILOVER_POLICY[$policyidx]}
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT | sed -e \
                "/Failover Policy: ${FAILOVER_POLICY[$policyidx]}/d"
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            echo " "
            echo " "

            policyidx=`expr $policyidx + 1`
        done
    fi


# Show configuration and status as appropriate for nodes in pool but not
# in any cluster.

    # Obtain the nodes in the pool
    get_pool_nodes
    if [ $? -ne 0 ]; then
        # Pool nodes could not be obtained. Exit.
        # Error message is printed and cleanup is done in
        # function get_pool_nodes.
        exit 1
    fi


    #If a pool node is not in any cluster print its details.
    nodeidx=0
    while [ $nodeidx -lt $NUMNODES ]
    do
        cnodeidx=0
        while [ $cnodeidx -lt $NUMCNODES ]
        do
            if [ ${NODE[$nodeidx]} = ${CNODE[$cnodeidx]} ]; then
                break
            fi
            cnodeidx=`expr $cnodeidx + 1`
        done

        if [ $cnodeidx -lt $NUMCNODES ]; then
            # Node in cluster. Skip node.
            nodeidx=`expr $nodeidx + 1`
            continue;
        fi

        # Node not in any cluster. Print status is desired and
        # configuration information for the node.

        echo "Pool_node ${NODE[$nodeidx]}:"
        echo " "

        # Print status of node if requested.
        if [ $SHOW_STATUS -eq 0 ]; then
            run_cmgr_command show status of node ${NODE[$nodeidx]}
            prependtab $CMGR_OUTPUT
            cat $CMGR_ERROR
            cat $CMGR_OUTPUT
            rm -f $CMGR_OUTPUT
            rm -f $CMGR_ERROR
            echo " "
        fi

        # Print configuration information for the node.
        run_cmgr_command show node ${NODE[$nodeidx]}
        prependtab $CMGR_OUTPUT
        cat $CMGR_ERROR
        cat $CMGR_OUTPUT
        rm -f $CMGR_OUTPUT
        rm -f $CMGR_ERROR
        echo " "

        echo " "

        nodeidx=`expr $nodeidx + 1`
    done
fi

# Cleanup and exit 0 (success).

rm -f $CMGR_OUTPUT
rm -f $CMGR_ERROR
rm -f $CMGR_OUTPUT1
rm -f $CMGR_ERROR1
rm -f $TMPFILE

exit 0